示例：点击 -> 性能监控

线上效果:

内存监控：

/proc/meminfo used=total-(buffers+cached+free)

[root@wangzi go]# cat /proc/meminfo
MemTotal:        1016212 kB
MemFree:           77724 kB
MemAvailable:      51692 kB
Buffers:            8468 kB
Cached:            59584 kB

python监控代码:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com

def mem_use():
    free=0
    used=0
    total=0
    buffers=0
    cached=0
    with open('/proc/meminfo','r') as file:
        for line in file.readlines():
            if line.startswith('MemTotal:'):
                total=line.split()[1]
            if line.startswith('MemFree:'):
                free=line.split()[1]
            if line.startswith('Cached:'):
                cached=line.split()[1]
            if line.startswith('Buffers:'):
                buffers=line.split()[1]
    used=int(total)-int(free)-int(cached)-int(buffers)
    insert_sql="""insert into `monitor_mem_use` (`used`,`free`,`cached`,`buffers`,`flow_time`) values ({0},{1},{2},{3},'{4}')""".format(used,free,cached,buffers,now_zero)
    cursorUpdate(insert_sql,[])

TCP连接监控:

/proc/net/tcp 第四列 01代表了 TCP_ESTABLISHED 06代表代表time_wait 08代表close_wait

[root@wangzi ~]# cat /proc/net/tcp| awk '{if($4 == '01') print $0}'|wc -l
22
[root@wangzi ~]# netstat -antpl|grep ESTABLISHED|wc -l
22
[root@wangzi ~]#cat /proc/net/tcp| awk '{if($4 == '01' || $4=='06' || $4=='08') print $4}'

因使用netstat命令有时会占用较多资源，当机器负载较高时，可以用使用上面的命令来查看tcp正在通信的连接数。

python监控:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com
def net_tcp():
    ret=Popen("""cat /proc/net/tcp| awk '{if($4 == '01' || $4=='06' || $4=='08') print $4}'""",shell=True,stdout=PIPE)
    established=0
    time_wait=0
    close_wait=0
    for item in ret.stdout.readlines():
       if int(item)==1:
           established+=1
       elif int(item)==6:
           time_wait+=1
       elif int(item)==6:
           close_wait+=1
    insert_sql="""insert into `monitor_net_tcp` (`establish`,`time_wait`,`close_wait`,`flow_time`) values ({0},{1},{2},'{3}')""".format(established,time_wait,close_wait,now_zero)
    cursorUpdate(insert_sql,[])

网卡IO监控:

通过/proc/net/dev获取网卡 IO,第1列和第9列分别代表了in 和out。

[root@wangzi go]#  cat /proc/net/dev
Inter-|   Receive                                                |  Transmit
face |bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
  eth0: 10020876640 52702968    0    0    0     0          0         0 11473861781 54530293    0    0    0     0       0          0
    lo: 6454958709 6187408    0    0    0     0          0         0 6454958709 6187408    0    0    0     0       0          0

python代码:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com
def write_net_io():
    net_in=0
    net_out=0
    add_net_in=0
    add_net_out=0
    with open('/proc/net/dev', 'r') as file:
        for line in file.readlines():
            if 'eth0' in line:
                net_in=int(line.split()[1])/1024
                net_out=int(line.split()[9])/1024
    select_sql="""select `in`,`out` from monitor_net_io order by id desc limit 1"""
    result=cursorQuery(select_sql,[])
    if len(result)>0:
        add_net_in=net_in-int(result[0][0])
        add_net_out=net_out-int(result[0][1])
    insert_sql="""insert into `monitor_net_io` (`in`,`out`,`add_in`,`add_out`,`flow_time`) values ({0},{1},{2},{3},'{4}')""".format(net_in,net_out,add_net_in,add_net_out,now_zero)
    cursorUpdate(insert_sql,[])

CPU监控:

[root@wangzi go]# cat /proc/stat
cpu  25187586 4339 20108620 1703341684 3875717 0 58452 0 0 0

pytho代码:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com
def cpu_use():
    use=0
    with open('/proc/stat','r') as file:
        for line in file.readlines():
            if line.startswith('cpu ',0,4):
                use=round((1-float(line.split()[4])/sum([int(x) for x in line.split()[1:]]))*100,5)
    insert_sql="""insert into `monitor_cpu_use` (`use`,`flow_time`) values ({0},'{1}')""".format(use,now_zero)
    cursorUpdate(insert_sql,[])

句柄监控:

/proc/sys/fs/file-nr 每列分别代表:

已分配文件句柄的数目已使用文件句柄的数目文件句柄的最大数目

[root@wangzi go]# cat /proc/sys/fs/file-nr
1280	0	98406

python监控代码:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com
def fd_use():
    free=0
    used=0
    total=0
    with open('/proc/sys/fs/file-nr','r') as file:
        for line in file.readlines():
            used=int(line.split()[0])
            total=int(line.split()[2])
    free=total-used
    insert_sql="""insert into `monitor_fd_use` (`used`,`free`,`flow_time`) values ({0},{1},'{2}')""".format(used,free,now_zero)
    cursorUpdate(insert_sql,[])

磁盘IO监控:

固定时间段监控/proc/vmstat 以下字段和前一个时间段相减，再除以时间间隔得到io。

[root@wangzi teg]# cat /proc/vmstat|grep pgpg
pgpgin 2094314778
pgpgout 265255600

python代码:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com
def write_disk_io():
    disk_in=0
    disk_out=0
    with open('/proc/vmstat', 'r') as file:
        for line in file.readlines():
            if 'pgpgin' in line:
                disk_in=int(line.split()[1])/1024
            if 'pgpgout' in line:
                disk_out=int(line.split()[1])/1024
    select_sql="""select `in`,`out` from monitor_disk_io order by id desc limit 1"""
    result=cursorQuery(select_sql,[])
    add_disk_in=disk_in-int(result[0][0])
    add_disk_out=disk_out-int(result[0][1])
    insert_sql="""insert into `monitor_disk_io` (`in`,`out`,`add_in`,`add_out`,`flow_time`) values ({0},{1},{2},{3},'{4}')""".format(disk_in,disk_out,add_disk_in,add_disk_out,now_zero)
    cursorUpdate(insert_sql,[])

分区空间使用率监控：

通过df命令统计：

[root@wangzi go]# df |grep -v ^Filesystem|awk -F ' ' '{print $6"|"$3"|"$4}'
/|6999556|42168392
/dev|0|498596
/dev/shm|24|508080
/run|412|507692
/sys/fs/cgroup|0|508104
/run/user/0|0|101624

python代码:

#!/usr/bin/env python
# coding=utf-8
# author: brownwang
# mail: 277215243@qq.com
# datetime:2019/3/31 1:03 PM
# web: https://www.bthlt.com
from subprocess import Popen,PIPE
def disk_df():
    ret=Popen("""df |grep -v ^Filesystem|awk -F ' ' '{print $6"|"$3"|"$4}'""",shell=True,stdout=PIPE)
    for item in ret.stdout.readlines():
        disk_info=item.replace('\n','').split('|')
        insert_sql="""insert into `monitor_disk_df` (`path`,`used`,`idle`,`flow_time`) values ('{0}',{1},{2},'{3}')""".format(disk_info[0],disk_info[1],disk_info[2],now_zero)
        cursorUpdate(insert_sql,[])

葫芦的运维日志

打赏